package com.example.spider.dianping;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.swing.*;
import java.awt.*;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.util.*;
import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * @date 19-9-23
 * @auther jackliang
 * @description TODO
 */

public class SpiderTool {


    public static String[] ua = {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 OPR/37.0.2178.32",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586",
            "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
            "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
            "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.3 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.277.400 QQBrowser/9.4.7658.400",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7",
            "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0",
    };


    public static void main(String[] args) throws Exception {
        JFrame frame = new JFrame("....");
        //文本组件长度30
        JTextField cityTextFieldName = new JTextField(10);
        //带默认内容的文本组件，长度10
        JTextField classificationTextField = new JTextField(10);

        JTextField addressTextField = new JTextField(10);
        JLabel cityLab = new JLabel("城市：");
        JLabel classificationLab = new JLabel("分类：");
        JLabel addressLab = new JLabel("地点：");

        //button

        JButton jButton = new JButton();
        jButton.addActionListener(new java.awt.event.ActionListener() {
            @Override
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                String classify = classificationTextField.getText() != null ? classificationTextField.getText() : "";
                String location = addressTextField.getText() != null ? addressTextField.getText() : "";


                if (cityTextFieldName.getText() == null || cityTextFieldName.getText().equals("")) {
                    System.out.println("city name is not null!");
                    return;
                }

                String region = cityTextFieldName.getText();
                // String urlPoint = "https://www.dianping.com/" + region  + classify +location;

                String urlPoint = "https://www.dianping.com/" + region + "/ch50";
//-------------------------------------------------------------------\\
                Document document;
                try {
                    document = Jsoup.connect(urlPoint).header("User-Agent", ua[new Random().nextInt(15)]).header("Upgrade-Insecure-Requests", "1").header("Host", "www.dianping.com").header("Connection", "keep-alive").header("Cache-Control", "max-age=0").header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3").cookie("Cookie", "navCtgScroll=0; showNav=#nav-tab|0|0; navCtgScroll=0; showNav=javascript:; _lxsdk_cuid=16d56d32781c8-08c21ed8dc19e1-30760d58-100200-16d56d32782c8; _lxsdk=16d56d32781c8-08c21ed8dc19e1-30760d58-100200-16d56d32782c8; _hc.v=59187b4b-5b20-1e62-b4cf-48f7bfc3f3ae.1569119742; s_ViewType=10; aburl=1; Hm_lvt_dbeeb675516927da776beeb1d9802bd4=1570520011; Hm_lpvt_dbeeb675516927da776beeb1d9802bd4=1570520011; cye=changsha; _lxsdk_s=16dab5295b4-1f3-ca7-848%7C0%7C171").execute().parse();
                } catch (IOException e) {
                    e.printStackTrace();
                    return;
                }
                List<String> shopids = new LinkedList<>();
                Map<String, String> classfy = new HashMap<>(18);
                Map<String, String> address = new HashMap<>(18);
                Element var = document.getElementById("classfy");
                Elements classfyVars = var.children();
                for (Element v1 : classfyVars) {
                    classfy.put(v1.text(), v1.attr("href").substring(v1.attr("href").lastIndexOf("/") + 1));
                }

                Element var1 = document.getElementById("J_nt_items");
                Elements eles = var1.children();
                for (Element v1 : eles) {
                    Elements eles2 = v1.children();
                    for (Element v3 : eles2) {
                        address.put(v3.text(), v3.attr("href").substring(v3.attr("href").lastIndexOf("/") + 1));
                    }
                }


                //------------------------------------------\\
                urlPoint = "http://www.dianping.com/" + region + "/ch50" + (classfy.get(classify) != null ? "/" + classfy.get(classify) : "") + address.getOrDefault(location, "");
                try {
                    document = Jsoup.connect(urlPoint).header("User-Agent", ua[new Random().nextInt(15)]).header("Upgrade-Insecure-Requests", "1").header("Host", "www.dianping.com").header("Connection", "keep-alive").header("Accept-Encoding", "gzip, deflate").header("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7").header("Cache-Control", "max-age=0").header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3").cookie("Cookie", "navCtgScroll=0; showNav=#nav-tab|0|0; navCtgScroll=0; showNav=javascript:; _lxsdk_cuid=16d56d32781c8-08c21ed8dc19e1-30760d58-100200-16d56d32782c8; _lxsdk=16d56d32781c8-08c21ed8dc19e1-30760d58-100200-16d56d32782c8; _hc.v=59187b4b-5b20-1e62-b4cf-48f7bfc3f3ae.1569119742; s_ViewType=10; aburl=1; Hm_lvt_dbeeb675516927da776beeb1d9802bd4=1570520011; Hm_lpvt_dbeeb675516927da776beeb1d9802bd4=1570520011; cye=changsha; _lxsdk_s=16dab5295b4-1f3-ca7-848%7C0%7C171").referrer("http://www.dianping.com/" + region + "/beauty").execute().parse();
                } catch (IOException e) {
                    e.printStackTrace();
                    return;
                }
                //--------------------------------------------\\
                Elements var12 = document.getElementsByClass("shop-list J_shop-list shop-all-list");
                Elements eless = var12.select("ul");
                Element ele = eless.get(0);
                Elements vars = ele.children();

                Elements pages = document.getElementsByClass("page");
                int pageNum;

                if (!pages.text().equals("")) {
                    Elements select = pages.select(".pageLink");
                    Element pageNumNode = select.last();
                    pageNum = Integer.parseInt(pageNumNode.text());
                    for (int i = 1; i <= pageNum; i++) {
                        if (i == 1) {
                            for (Element var111 : vars) {
                                String var3 = var111.getElementsByAttributeValue("data-click-name", "shop_tag_region_click").
                                        get(0).
                                        attr("data-shopid");
                                shopids.add(var3);
                            }
                        } else {

                            //第2 3 。。。页
                            String pageUrl = urlPoint + "p" + i;
                            Document var_doc;
                            var_doc = getDocument(pageUrl, region, i);

                            if (var_doc == null) {
                                System.out.println("在访问" + pageUrl + "的第" + i + "页的时候,请求被拒绝,请稍后尝试！");
                                return;
                            }

                            Elements var12_ = var_doc.getElementsByClass("shop-list J_shop-list shop-all-list");
                            Elements eless_ = var12_.select("ul");
                            Element ele_ = eless_.get(0);
                            Elements vars_ = ele_.children();
                            for (Element val_1v : vars_) {
                                String var3 = val_1v.getElementsByAttributeValue("data-click-name", "shop_tag_region_click").
                                        get(0).
                                        attr("data-shopid");
                                shopids.add(var3);
                            }
                        }
                    }
                } else {
                    for (Element var111 : vars) {
                        String var3 = var111.getElementsByAttributeValue("data-click-name", "shop_tag_region_click").
                                get(0).
                                attr("data-shopid");
                        shopids.add(var3);
                    }
                }
//
                System.out.println("正在获取一共" + shopids.size() + "家店铺信息,请稍后。。。。");
                Random random = new Random( );
                File tempFile = null;
                File file;
                String curSys = System.getProperty("os.name").toLowerCase();
                System.out.printf("当前系统为:%s.当前给定生成路经为:%s\n", curSys, args[0]);


                if (curSys.equals("windows")) {
                    file = new File(args[0]);
                } else {
                    file = new File(args[0]);
                }

                if (!file.exists()){
                    System.out.printf("您输入的路经在当前"+curSys+"系统不存在,请重新输入");
                    return;
                }

                BufferedWriter bufferedWriter;
                try {
                    tempFile = Files.createTempFile(file.toPath(), random.nextLong() + "", ".txt").toFile();
                    bufferedWriter = new BufferedWriter(new FileWriter(tempFile));
                } catch (AccessDeniedException e) {
                    e.printStackTrace();
                    System.out.println("在您所提供的目录下创建，不符合您权限范畴，请检查您是否是超级管理员.");
                    return;
                }catch (Exception e){
                    e.printStackTrace();
                    return;
                }

                //---------------------------------------------------------------------------\\
                int ii = 0;
                for (String shopId : shopids) {

                    String url1 = "http://www.dianping.com/shop/" + shopId;
                    Document doc;
                    Connection.Response response = null;
                    try {
                        response = Jsoup.connect(url1).execute();
                    } catch (Exception e) {
                        e.printStackTrace();
                    }

                    try {
                        doc = response.parse();
                    } catch (Exception e) {
                        e.printStackTrace();
                        return;
                    }
                    Elements elements11 = doc.getElementsByClass("shop-name");
                    String shopName = elements11.text();//28155
                    String shopNameValue = shopName.substring(0, shopName.lastIndexOf(" "));
                    Elements elements111 = doc.getElementsByAttributeValue("itemprop", "tel");
                    String tel = elements111.text();
                    String val = shopNameValue + "," + tel;

                    try {
                        bufferedWriter.write(val);
                        System.out.println("line=\t" + val + "当前剩余" + (shopids.size() - (++ii)) + "个店铺");
                        bufferedWriter.newLine();
                        TimeUnit.MILLISECONDS.sleep(400);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
                try {
                    bufferedWriter.flush();
                } catch (IOException e) {
                }
            }


            private Document getDocument(String pageUrl, String region, int i) {
                Document document = null;
                boolean fail;
                int MAX = 10;
                int x = 0;
                while (true) {
                    System.out.println("页码" + i + "重试第" + x + "次");
                    if (x > MAX) {
                        fail = true;
                        break;
                    }
                    try {
                        document = Jsoup.connect(pageUrl).header("User-Agent", ua[new Random().nextInt(15)]).header("Upgrade-Insecure-Requests", "1").header("Host", "www.dianping.com").header("Connection", "keep-alive").header("Accept-Encoding", "gzip, deflate").header("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7").header("Cache-Control", "max-age=0").header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3").cookie("Cookie", "navCtgScroll=0; showNav=#nav-tab|0|0; navCtgScroll=0; showNav=javascript:; _lxsdk_cuid=16d56d32781c8-08c21ed8dc19e1-30760d58-100200-16d56d32782c8; _lxsdk=16d56d32781c8-08c21ed8dc19e1-30760d58-100200-16d56d32782c8; _hc.v=59187b4b-5b20-1e62-b4cf-48f7bfc3f3ae.1569119742; s_ViewType=10; aburl=1; Hm_lvt_dbeeb675516927da776beeb1d9802bd4=1570520011; Hm_lpvt_dbeeb675516927da776beeb1d9802bd4=1570520011; cye=" + region + "; _lxsdk_s=16dab5295b4-1f3-ca7-848%7C0%7C171").referrer("www.dianping.com").execute().parse();
                        System.out.println("此时页码为=" + i);
                        return document;
                    } catch (Exception e) {
                        e.printStackTrace();
                        try {
                            TimeUnit.MILLISECONDS.sleep(3000);
                        } catch (InterruptedException e1) {
                            e1.printStackTrace();
                        }
                        System.err.println("HTTP error fetching URL. Status=403,现在正在重试");
                    } finally {
                        x++;
                    }
                }
                if (fail) {
                    return null;
                }
                return document;
            }

        });

        jButton.setLocation(100, 100);
        //设置组件的长度
        cityTextFieldName.setColumns(10);
        classificationTextField.setColumns(10);
        //设置默认文本组件不可编辑
//        noed.setEnabled(false);
        //使用网格布局
        frame.add(jButton);
        frame.setLayout(new GridLayout(5, 5));
        frame.add(cityLab);
        frame.add(cityTextFieldName);
        frame.add(classificationLab);
        frame.add(classificationTextField);
        frame.add(addressLab);
        frame.add(addressTextField);
        frame.setSize(400, 100);
        frame.setLocation(300, 200);
        frame.setVisible(true);
    }


}
